import requests
from requests.exceptions import RequestException
from urllib.parse import urlencode
import json
from bs4 import BeautifulSoup 
import re
import time
import os
from hashlib import md5
from multiprocessing import Pool
from config import *
import pymongo

client=pymongo.MongoClient(MONGO_URL,connect=False)
db=client[MONGO_DB]

def getOnePpageIndex(offset, key):
    data = {
        'autoload': 'true',
        'count': 20,
        'cur_tab': 3,
        'format': 'json',
        'keyword': key,
        'offset': offset,
		'from':'gallery'
    }
    params = urlencode(data)
    base = 'http://www.toutiao.com/search_content/'
    url = base + '?' + params
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
        return None
    except ConnectionError:
        print('Error occurred')
        return None

def jsonToStr(html):
	data=json.loads(html)
	if data and 'data' in data.keys():
		for item in data.get('data'):
			yield item.get('article_url')

def getDeatilPage(url):
	try:
		response = requests.get(url,timeout=600)
		if response.status_code == 200:
			return response.text
		return None
	except requests.exceptions.ConnectionError:
		print('Error occurred')
		return None
	

def parseDetailPage(html,url):
	soup=BeautifulSoup(html,'lxml')
	title=soup.select('title')[0].get_text()
	image=re.compile('gallery: JSON.parse\("(.*?)"\),',re.S)
	result=re.search(image,html)
	if result:
		data = json.loads(result.group(1).replace('\\', ''))
		if data and 'sub_images' in data.keys():
			subImages=data.get('sub_images')
			images=[item.get('url') for item in subImages]
			for image in images: downLoadImage(image)
			return {
				'title':title,
				'images':images,
				'url':url
			}
		
def downLoadImage(url):
	print('正在下载:  ',url)
	try:
		response = requests.get(url,timeout=600)
		if response.status_code == 200:
			return saveImage(response.content)
		return None
	except requests.exceptions.ConnectionError:
		print('请求图片出错')
		return None

def saveImage(content):
	path = '{0}/{1}.{2}'.format(os.getcwd(), md5(content).hexdigest(), 'jpg')
	if not os.path.exists(path):
		with open(path,'wb') as f:
			f.write(content)
			f.close


def saveToMonogo(result):
	if db[MONGO_TABLE].insert(result):
		print('存储到MongoDB成功',result)
		return True
	return False



def main(offset):
	text = getOnePpageIndex(0, '街拍')
	urls=jsonToStr(text)
	for url in urls:
		time.sleep(1)
		html=getDeatilPage(url)
		if html:
			result=parseDetailPage(html,url)
			if result:
				saveToMonogo(result)
			



if __name__ == '__main__':
    pool = Pool()
    groups = ([x * 20 for x in range(1, 20 + 1)])
    pool.map(main, groups)
    pool.close()
    pool.join()